{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bc2bc6f3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mesolitica/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('mistralai/Mistral-Small-3.1-24B-Instruct-2503')\n",
    "x = {\n",
    "  \"chat_template\": \"{%- set today = strftime_now(\\\"%Y-%m-%d\\\") %}\\n{%- set default_system_message = \\\"You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\\\nYour knowledge base was last updated on 2023-10-01. The current date is \\\" + today + \\\".\\\\n\\\\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\\\\nIf the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \\\\\\\"What are some good restaurants around me?\\\\\\\" => \\\\\\\"Where are you?\\\\\\\" or \\\\\\\"When is the next flight to Tokyo\\\\\\\" => \\\\\\\"Where do you travel from?\\\\\\\")\\\" %}\\n\\n{{- bos_token }}\\n\\n{%- if messages[0]['role'] == 'system' %}\\n    {%- if messages[0]['content'] is string %}\\n        {%- set system_message = messages[0]['content'] %}\\n    {%- else %}\\n        {%- set system_message = messages[0]['content'][0]['text'] %}\\n    {%- endif %}\\n    {%- set loop_messages = messages[1:] %}\\n{%- else %}\\n    {%- set system_message = default_system_message %}\\n    {%- set loop_messages = messages %}\\n{%- endif %}\\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\\n\\n{%- for message in loop_messages %}\\n    {%- if message['role'] == 'user' %}\\n        {%- if message['content'] is string %}\\n            {{- '[INST]' + message['content'] + '[/INST]' }}\\n        {%- else %}\\n            {{- '[INST]' }}\\n            {%- for block in message['content'] %}\\n                {%- if block['type'] == 'text' %}\\n                    {{- block['text'] }}\\n                {%- elif block['type'] in ['image', 'image_url'] %}\\n                    {{- '[IMG]' }}\\n                {%- else %}\\n                    {{- raise_exception('Only text and image blocks are supported in message content!') }}\\n                {%- endif %}\\n            {%- endfor %}\\n            {{- '[/INST]' }}\\n        {%- endif %}\\n    {%- elif message['role'] == 'system' %}\\n        {%- if message['content'] is string %}\\n            {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}\\n        {%- else %}\\n            {{- '[SYSTEM_PROMPT]' + message['content'][0]['text'] + '[/SYSTEM_PROMPT]' }}\\n        {%- endif %}\\n    {%- elif message['role'] == 'assistant' %}\\n        {%- if message['content'] is string %}\\n            {{- message['content'] + eos_token }}\\n        {%- else %}\\n            {{- message['content'][0]['text'] + eos_token }}\\n        {%- endif %}\\n    {%- else %}\\n        {{- raise_exception('Only user, system and assistant roles are supported!') }}\\n    {%- endif %}\\n{%- endfor %}\"\n",
    "}\n",
    "tokenizer.chat_template = x['chat_template']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf63d9f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "wget https://huggingface.co/datasets/mesolitica/Malaysian-SFT/resolve/main/combine/combined-malaysian-reasoning.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0a89e7f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = {\n",
    "  \"chat_template\": \"{%- set today = strftime_now(\\\"%Y-%m-%d\\\") %}\\n{%- set default_system_message = \\\"You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\\\nYour knowledge base was last updated on 2023-10-01. The current date is \\\" + today + \\\".\\\\n\\\\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\\\\nIf the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \\\\\\\"What are some good restaurants around me?\\\\\\\" => \\\\\\\"Where are you?\\\\\\\" or \\\\\\\"When is the next flight to Tokyo\\\\\\\" => \\\\\\\"Where do you travel from?\\\\\\\")\\\" %}\\n\\n{{- bos_token }}\\n\\n{%- if messages[0]['role'] == 'system' %}\\n    {%- if messages[0]['content'] is string %}\\n        {%- set system_message = messages[0]['content'] %}\\n    {%- else %}\\n        {%- set system_message = messages[0]['content'][0]['text'] %}\\n    {%- endif %}\\n    {%- set loop_messages = messages[1:] %}\\n{%- else %}\\n    {%- set system_message = default_system_message %}\\n    {%- set loop_messages = messages %}\\n{%- endif %}\\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\\n\\n{%- for message in loop_messages %}\\n    {%- if message['role'] == 'user' %}\\n        {%- if message['content'] is string %}\\n            {{- '[INST]' + message['content'] + '[/INST]' }}\\n        {%- else %}\\n            {{- '[INST]' }}\\n            {%- for block in message['content'] %}\\n                {%- if block['type'] == 'text' %}\\n                    {{- block['text'] }}\\n                {%- elif block['type'] in ['image', 'image_url'] %}\\n                    {{- '[IMG]' }}\\n                {%- else %}\\n                    {{- raise_exception('Only text and image blocks are supported in message content!') }}\\n                {%- endif %}\\n            {%- endfor %}\\n            {{- '[/INST]' }}\\n        {%- endif %}\\n    {%- elif message['role'] == 'system' %}\\n        {%- if message['content'] is string %}\\n            {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}\\n        {%- else %}\\n            {{- '[SYSTEM_PROMPT]' + message['content'][0]['text'] + '[/SYSTEM_PROMPT]' }}\\n        {%- endif %}\\n    {%- elif message['role'] == 'assistant' %}\\n        {%- if message['content'] is string %}\\n            {{- message['content'] + eos_token }}\\n        {%- else %}\\n            {{- message['content'][0]['text'] + eos_token }}\\n        {%- endif %}\\n    {%- else %}\\n        {{- raise_exception('Only user, system and assistant roles are supported!') }}\\n    {%- endif %}\\n{%- endfor %}\"\n",
    "}\n",
    "tokenizer.chat_template = x['chat_template']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e0101f87",
   "metadata": {},
   "outputs": [],
   "source": [
    "from streaming import MDSWriter\n",
    "from streaming.base.format.mds.encodings import Encoding, _encodings\n",
    "from streaming import LocalDataset\n",
    "import streaming\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "import os\n",
    "import json\n",
    "\n",
    "class UInt32(Encoding):\n",
    "    def encode(self, obj) -> bytes:\n",
    "        return obj.tobytes()\n",
    "\n",
    "    def decode(self, data: bytes):\n",
    "        return np.frombuffer(data, np.uint32)\n",
    "\n",
    "_encodings['uint32'] = UInt32\n",
    "\n",
    "columns = {\n",
    "    'input_ids': 'uint32',\n",
    "    'position_ids': 'uint32',\n",
    "    'attention_mask': 'uint32',\n",
    "}\n",
    "hashes = 'sha1', 'xxh64'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "73655b5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf tokenized-8k\n",
    "!mkdir tokenized-8k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cb2d8c3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1327479"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine = []\n",
    "with open('../combined-malaysian-sft.jsonl') as fopen:\n",
    "    for l in fopen:\n",
    "        l = json.loads(l)\n",
    "        combine.append(l)\n",
    "\n",
    "len(combine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "841403fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1397479"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../translation-instructions.json') as fopen:\n",
    "    translation = json.load(fopen)\n",
    "    \n",
    "for d in translation:\n",
    "    combine.append([\n",
    "        {'role': 'user', 'content': d['input']},\n",
    "        {'role': 'assistant', 'content': d['output']}\n",
    "    ])\n",
    "    \n",
    "len(combine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "03fb8477",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../combined-malaysian-reasoning.jsonl') as fopen:\n",
    "    for l in fopen:\n",
    "        l = json.loads(l)\n",
    "        combine.append(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "15d5986d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'role': 'system',\n",
       "  'content': 'You are going to enter reasoning mode, first you try to think step-by-step in malay after that give the final answer.'},\n",
       " {'role': 'user',\n",
       "  'content': 'Mungkin bagi mak ayah aku biasa je.\\n\\nterjemah ke standard english'},\n",
       " {'role': 'assistant',\n",
       "  'content': 'Okay, mari kita pecahkan ayat Melayu tempatan ini dan ubahnya menjadi bahasa Inggeris standard langkah demi langkah.\\n\\n## Langkah Demi Langkah: Terjemahan Ayat Melayu Tempatan ke Bahasa Inggeris Standard\\n\\nAyat Melayu tempatan:\\n**\"Mungkin bagi mak ayah aku biasa je.\"**\\n\\nMari kita fahami dan pecahkan ayat ini untuk diterjemah dengan tepat.\\n\\n### Langkah 1: Fahami Maksud Keseluruhan Ayat\\n\\nAyat ini menyatakan bahawa sesuatu perkara mungkin dianggap biasa sahaja oleh ibu bapa penutur. Nada ayat ini agak santai dan menggunakan kata-kata yang ringkas.\\n\\n### Langkah 2: Pecahkan Ayat kepada Bahagian Kecil\\n\\nAyat ini boleh dipisahkan kepada beberapa bahagian:\\n\\n1. \"Mungkin\" - mungkin, probable\\n2. \"bagi mak ayah aku\" - for my mother and father, or simply for my parents\\n3. \"biasa je\" - just normal, only normal\\n\\n### Langkah 3: Tukar Kata-Kata Tempatan kepada Bahasa Melayu Standard\\n\\nSekarang kita akan tukar setiap frasa kepada Bahasa Melayu standard:\\n\\n- \"Mungkin\" sudah standard, boleh kekalkan.\\n- \"mak ayah aku\" ialah istilah tempatan untuk merujuk kepada ibu bapa. Dalam Bahasa Melayu standard, ia menjadi \"ibu bapa saya\" atau \"orang tua saya\".\\n- \"biasa je\" ialah frasa santai untuk \"biasa sahaja\". Dalam bentuk standard, ia menjadi \"sahaja biasa\" atau \"hanya biasa\".\\n\\nJadi ayat Bahasa Melayu standard penuh akan menjadi:\\n\\n**\"Mungkin bagi ibu bapa saya, hanya biasa.\"**\\n\\n### Langkah 4: Terjemah ke Bahasa Inggeris Standard\\n\\nSeterusnya, kita akan terjemahkan ayat Bahasa Melayu standard ini ke dalam Bahasa Inggeris.\\n\\n- \"Mungkin\" -> \"Maybe\" atau \"Perhaps\"\\n- \"bagi ibu bapa saya\" -> \"for my parents\"\\n- \"hanya biasa\" -> \"nothing\" atau \"just ordinary\"\\n\\nJadi terjemahan Bahasa Inggeris standard:\\n\\n**\"Maybe for my parents, that is nothing.\"**\\n\\nAtau boleh ditulis sebagai:\\n\\n**\"Perhaps for my parents, that is just ordinary.\"**\\n\\n### Langkah 5: Prinsip Umum yang Digunakan dalam Proses Ini\\n\\nBerikut adalah beberapa prinsip penting yang patut diingati apabila menukar ayat Melayu tempatan kepada Bahasa Inggeris standard:\\n\\n1. **Pemahaman Konteks:** Fahami apa yang dimaksudkan dengan ayat itu, bukan hanya perkataan-perkataan individu.\\n2. **Penggantian Frasa Tempatan:** Tukar frasa tempatan yang kurang formal kepada versi yang lebih standard.\\n3. **Penyelarasan Struktur Ayat:** Sesuaikan struktur ayat supaya ia kedengaran semulajadi dalam Bahasa Inggeris.\\n4. **Ketepatan Makna:** Pastikan terjemahan itu menyampaikan makna asal dengan tepat.\\n\\n**Jawapan Akhir**\\n\\n$boxed{Maybe for my parents, that is nothing.}$.'}]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8c0d2e8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "row = combine[-1]\n",
    "prompt = tokenizer.apply_chat_template(row, tokenize=False)\n",
    "outputs = tokenizer(prompt, add_special_tokens = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "679359c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<s>[SYSTEM_PROMPT]You are going to enter reasoning mode, first you try to think step-by-step in malay after that give the final answer.[/SYSTEM_PROMPT][INST]Mungkin bagi mak ayah aku biasa je.\\n\\nterjemah ke standard english[/INST]Okay, mari kita pecahkan ayat Melayu tempatan ini dan ubahnya menjadi bahasa Inggeris standard langkah demi langkah.\\n\\n## Langkah Demi Langkah: Terjemahan Ayat Melayu Tempatan ke Bahasa Inggeris Standard\\n\\nAyat Melayu tempatan:\\n**\"Mungkin bagi mak ayah aku biasa je.\"**\\n\\nMari kita fahami dan pecahkan ayat ini untuk diterjemah dengan tepat.\\n\\n### Langkah 1: Fahami Maksud Keseluruhan Ayat\\n\\nAyat ini menyatakan bahawa sesuatu perkara mungkin dianggap biasa sahaja oleh ibu bapa penutur. Nada ayat ini agak santai dan menggunakan kata-kata yang ringkas.\\n\\n### Langkah 2: Pecahkan Ayat kepada Bahagian Kecil\\n\\nAyat ini boleh dipisahkan kepada beberapa bahagian:\\n\\n1. \"Mungkin\" - mungkin, probable\\n2. \"bagi mak ayah aku\" - for my mother and father, or simply for my parents\\n3. \"biasa je\" - just normal, only normal\\n\\n### Langkah 3: Tukar Kata-Kata Tempatan kepada Bahasa Melayu Standard\\n\\nSekarang kita akan tukar setiap frasa kepada Bahasa Melayu standard:\\n\\n- \"Mungkin\" sudah standard, boleh kekalkan.\\n- \"mak ayah aku\" ialah istilah tempatan untuk merujuk kepada ibu bapa. Dalam Bahasa Melayu standard, ia menjadi \"ibu bapa saya\" atau \"orang tua saya\".\\n- \"biasa je\" ialah frasa santai untuk \"biasa sahaja\". Dalam bentuk standard, ia menjadi \"sahaja biasa\" atau \"hanya biasa\".\\n\\nJadi ayat Bahasa Melayu standard penuh akan menjadi:\\n\\n**\"Mungkin bagi ibu bapa saya, hanya biasa.\"**\\n\\n### Langkah 4: Terjemah ke Bahasa Inggeris Standard\\n\\nSeterusnya, kita akan terjemahkan ayat Bahasa Melayu standard ini ke dalam Bahasa Inggeris.\\n\\n- \"Mungkin\" -> \"Maybe\" atau \"Perhaps\"\\n- \"bagi ibu bapa saya\" -> \"for my parents\"\\n- \"hanya biasa\" -> \"nothing\" atau \"just ordinary\"\\n\\nJadi terjemahan Bahasa Inggeris standard:\\n\\n**\"Maybe for my parents, that is nothing.\"**\\n\\nAtau boleh ditulis sebagai:\\n\\n**\"Perhaps for my parents, that is just ordinary.\"**\\n\\n### Langkah 5: Prinsip Umum yang Digunakan dalam Proses Ini\\n\\nBerikut adalah beberapa prinsip penting yang patut diingati apabila menukar ayat Melayu tempatan kepada Bahasa Inggeris standard:\\n\\n1. **Pemahaman Konteks:** Fahami apa yang dimaksudkan dengan ayat itu, bukan hanya perkataan-perkataan individu.\\n2. **Penggantian Frasa Tempatan:** Tukar frasa tempatan yang kurang formal kepada versi yang lebih standard.\\n3. **Penyelarasan Struktur Ayat:** Sesuaikan struktur ayat supaya ia kedengaran semulajadi dalam Bahasa Inggeris.\\n4. **Ketepatan Makna:** Pastikan terjemahan itu menyampaikan makna asal dengan tepat.\\n\\n**Jawapan Akhir**\\n\\n$boxed{Maybe for my parents, that is nothing.}$.</s>'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5f4b922a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "\n",
    "def collator(batch, batch_position_ids):\n",
    "    input_ids = []\n",
    "    position_ids = []\n",
    "    masks = []\n",
    "    for i in range(len(batch)):\n",
    "        l = len(batch[i])\n",
    "        input_ids.extend(batch[i])\n",
    "        position_ids.extend(batch_position_ids[i])\n",
    "        masks.append(l)\n",
    "    \n",
    "    return {\n",
    "        'input_ids': np.array(input_ids).astype(np.uint32),\n",
    "        'position_ids': np.array(position_ids).astype(np.uint32),\n",
    "        'attention_mask': np.array(masks).astype(np.uint32),\n",
    "    }\n",
    "\n",
    "def slice_and_balance(nested_list, size):\n",
    "    first = []\n",
    "    balance = []\n",
    "    current_size = 0\n",
    "\n",
    "    for sublist in nested_list:\n",
    "        if current_size < size:\n",
    "            remaining_space = size - current_size\n",
    "            if len(sublist) <= remaining_space:\n",
    "                first.append(sublist)\n",
    "                current_size += len(sublist)\n",
    "            else:\n",
    "                first.append(sublist[:remaining_space])\n",
    "                balance.append(sublist[remaining_space:])\n",
    "                current_size = size\n",
    "        else:\n",
    "            balance.append(sublist)\n",
    "    \n",
    "    return first, balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f38bbff1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def loop(files, block_size = 8192):\n",
    "    rows, index = files\n",
    "    out_root = f'tokenized-8k/tokenized-{index}'\n",
    "    os.system(f'rm -rf {out_root}')\n",
    "    count = 0\n",
    "    temp = []\n",
    "    position_ids = []\n",
    "    last_block, last_position_block = None, None\n",
    "    with MDSWriter(out=out_root, columns=columns, compression=None, hashes=hashes) as out:\n",
    "        for row in tqdm(rows):\n",
    "            prompt = tokenizer.apply_chat_template(row, tokenize=False)\n",
    "            outputs = tokenizer(prompt, add_special_tokens = False)\n",
    "            temp.append(outputs['input_ids'])\n",
    "            position_ids.append(range(len(outputs['input_ids'])))\n",
    "            count += len(outputs['input_ids'])\n",
    "            while count >= block_size:\n",
    "                block, temp = slice_and_balance(temp, block_size)\n",
    "                block_position, position_ids = slice_and_balance(position_ids, block_size)\n",
    "                count = count - block_size\n",
    "                o = collator(block, block_position)\n",
    "                last_block = block\n",
    "                last_position_block = block_position\n",
    "                out.write(o)\n",
    "                \n",
    "        block, _ = slice_and_balance(last_block, block_size - count)\n",
    "        block_position, _ = slice_and_balance(last_position_block, block_size - count)\n",
    "\n",
    "        block.extend(temp)\n",
    "        block_position.extend(position_ids)\n",
    "\n",
    "        o = collator(block, block_position)\n",
    "        if len(o['input_ids']) == block_size:\n",
    "            out.write(o)\n",
    "            return o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ccf09d78",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 1000/1000 [00:01<00:00, 589.27it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input_ids': array([30114,  1816, 45191, ...,  1271,  1046,     2], dtype=uint32),\n",
       " 'position_ids': array([791, 792, 793, ..., 474, 475, 476], dtype=uint32),\n",
       " 'attention_mask': array([490, 703, 320, 748, 413, 698, 662, 962, 852, 476, 612, 779, 477],\n",
       "       dtype=uint32)}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loop((combine[:1000], 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "834a6704",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "  5%|████                                                                             | 2478/50000 [00:04<01:14, 640.26it/s]TOKENIZERS_PARALLELISM=(true | false)\n",
      "  8%|██████▏                                                                          | 3804/50000 [00:11<02:17, 336.22it/s]TOKENIZERS_PARALLELISM=(true | false)\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [00:54<00:00, 915.00it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:39<00:00, 504.56it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████| 50000/50000 [00:43<00:00, 1158.58it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:25<00:00, 587.98it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:58<00:00, 422.06it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:39<00:00, 503.15it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████| 50000/50000 [00:38<00:00, 1304.98it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████| 50000/50000 [00:38<00:00, 1292.84it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:13<00:00, 374.69it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:34<00:00, 322.93it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████| 50000/50000 [00:44<00:00, 1128.05it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:33<00:00, 325.42it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:27<00:00, 570.72it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [03:05<00:00, 269.04it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [03:30<00:00, 237.52it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:52<00:00, 444.48it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:08<00:00, 388.77it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:34<00:00, 528.96it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:04<00:00, 402.80it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:52<00:00, 445.95it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 3544/3544 [00:10<00:00, 335.76it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:35<00:00, 322.46it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:38<00:00, 508.70it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:36<00:00, 318.84it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:01<00:00, 809.46it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [01:10<00:00, 712.61it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:17<00:00, 363.08it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [02:53<00:00, 287.66it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 50000/50000 [03:46<00:00, 220.43it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 50000/50000 [08:49<00:00, 94.51it/s]\n"
     ]
    }
   ],
   "source": [
    "from multiprocess import Pool\n",
    "\n",
    "def chunks(l, n):\n",
    "    for i in range(0, len(l), n):\n",
    "        yield (l[i: i + n], i // n)\n",
    "\n",
    "chunks = chunks(combine, 50000)\n",
    "pool = Pool(10)\n",
    "pooled = pool.map(loop, chunks)\n",
    "pool.close()\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "31989c6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tokenized-8k/tokenized-0',\n",
       " 'tokenized-8k/tokenized-1',\n",
       " 'tokenized-8k/tokenized-2',\n",
       " 'tokenized-8k/tokenized-3',\n",
       " 'tokenized-8k/tokenized-4',\n",
       " 'tokenized-8k/tokenized-5',\n",
       " 'tokenized-8k/tokenized-6',\n",
       " 'tokenized-8k/tokenized-7',\n",
       " 'tokenized-8k/tokenized-8',\n",
       " 'tokenized-8k/tokenized-9',\n",
       " 'tokenized-8k/tokenized-10',\n",
       " 'tokenized-8k/tokenized-11',\n",
       " 'tokenized-8k/tokenized-12',\n",
       " 'tokenized-8k/tokenized-13',\n",
       " 'tokenized-8k/tokenized-14',\n",
       " 'tokenized-8k/tokenized-15',\n",
       " 'tokenized-8k/tokenized-16',\n",
       " 'tokenized-8k/tokenized-17',\n",
       " 'tokenized-8k/tokenized-18',\n",
       " 'tokenized-8k/tokenized-19',\n",
       " 'tokenized-8k/tokenized-20',\n",
       " 'tokenized-8k/tokenized-21',\n",
       " 'tokenized-8k/tokenized-22',\n",
       " 'tokenized-8k/tokenized-23',\n",
       " 'tokenized-8k/tokenized-24',\n",
       " 'tokenized-8k/tokenized-25',\n",
       " 'tokenized-8k/tokenized-26',\n",
       " 'tokenized-8k/tokenized-27',\n",
       " 'tokenized-8k/tokenized-28',\n",
       " 'tokenized-8k/tokenized-29']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "folders = sorted(glob('tokenized-8k/tokenized-*'), key = lambda x: int(x.split('-')[-1]))\n",
    "folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "25c37842",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!rm -rf packing-8k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4f1f53bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 4676/4676 [00:00<00:00, 5304.68it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 2383/2383 [00:00<00:00, 5769.92it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 8955/8955 [00:01<00:00, 4960.79it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 5687/5687 [00:01<00:00, 4780.56it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 6295/6295 [00:01<00:00, 4962.19it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 6793/6793 [00:01<00:00, 4766.24it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 9526/9526 [00:01<00:00, 5276.49it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 8502/8502 [00:01<00:00, 5044.87it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 4432/4432 [00:00<00:00, 5543.66it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 3769/3769 [00:00<00:00, 4537.18it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 1879/1879 [00:00<00:00, 4796.52it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 1572/1572 [00:00<00:00, 4258.47it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 1574/1574 [00:00<00:00, 7099.12it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 1897/1897 [00:00<00:00, 4833.73it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 4297/4297 [00:00<00:00, 5292.11it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 5968/5968 [00:01<00:00, 4950.86it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 5140/5140 [00:01<00:00, 5080.25it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 5664/5664 [00:01<00:00, 4909.45it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 7034/7034 [00:01<00:00, 5782.62it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 7190/7190 [00:01<00:00, 5264.06it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 4824/4824 [00:01<00:00, 4805.25it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 4139/4139 [00:00<00:00, 5224.61it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 8023/8023 [00:01<00:00, 5012.76it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 4191/4191 [00:00<00:00, 5242.05it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 6671/6671 [00:01<00:00, 4930.10it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████| 12104/12104 [00:02<00:00, 5363.45it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 2760/2760 [00:00<00:00, 4866.45it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 3047/3047 [00:00<00:00, 5191.74it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████| 24541/24541 [00:04<00:00, 5301.87it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████| 508/508 [00:00<00:00, 2872.76it/s]\n"
     ]
    }
   ],
   "source": [
    "with MDSWriter(out='packing-8k', columns=columns, compression=None, hashes=hashes) as out:\n",
    "    for f in folders:\n",
    "        try:\n",
    "            dataset = LocalDataset(local=f)\n",
    "            for i in tqdm(range(len(dataset))):\n",
    "                out.write(dataset[i])\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1a00ecf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = LocalDataset(local='packing-8k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2601bda7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.425743872"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset) * 8192 / 1e9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b7b438ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Cari bucu bagi fungsi nilai mutlak f(x) = |2x - 5|.[/INST]Untuk mencari bucu bagi fungsi nilai mutlak f(x) = |2x - 5|, kita perlu mencari koordinat-x bagi puncak tersebut. Puncak berlaku pada titik di mana ungkapan di dalam nilai mutlak adalah sama dengan sifar.\\n\\n2x - 5 = 0\\n\\nSekarang, selesaikan untuk x:\\n\\n2x = 5\\nx = 5/2\\n\\nKoordinat-x bagi bucu ialah 5/2. Sekarang, kita perlu mencari koordinat-y yang sepadan dengan memasukkan semula koordinat-x ke dalam fungsi:\\n\\nf(5/2) = |2(5/2) - 5|\\n\\nf(5/2) = |5 - 5|\\n\\nf(5/2) = 0\\n\\nJadi, koordinat y bagi bucu ialah 0. Oleh itu, bucu bagi fungsi nilai mutlak f(x) = |2x - 5| ialah (5/2, 0).</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Biarkan $f:\\\\mathbb{R}^2 \\\\to \\\\mathbb{R}$ ditakrifkan oleh $f(x,y) = x^2 + y^2$. Cari semua titik kritikal $f$ dan tentukan sama ada ia adalah maksima tempatan, minimum tempatan atau titik pelana menggunakan teori Morse.[/INST]Untuk mencari titik genting bagi fungsi $f(x, y) = x^2 + y^2$, pertama kita perlu mengira kecerunannya, yang terdiri daripada terbitan separa berkenaan dengan $x$ dan $y$:\\n\\n$\\\\frac{\\\\partial f}{\\\\partial x} = 2x$\\n\\n$\\\\frac{\\\\partial f}{\\\\partial y} = 2y$\\n\\nTitik kritikal berlaku apabila kecerunan adalah sama dengan vektor sifar, iaitu, apabila kedua-dua terbitan separa adalah sama dengan sifar:\\n\\n$2x = 0 \\\\Anak panah kanan x = 0$\\n\\n$2y = 0 \\\\Anak panah kanan y = 0$\\n\\nOleh itu, hanya terdapat satu titik kritikal, $(0, 0)$.\\n\\nSekarang, kita akan menggunakan teori Morse untuk mengklasifikasikan titik kritikal ini. Untuk berbuat demikian, kita perlu mengira matriks Hessian bagi $f$, iaitu matriks terbitan separa kedua:\\n\\n$H(f) = \\\\begin{bmatrix}\\n\\\\frac{\\\\partial^2 f}{\\\\partial x^2} & \\\\frac{\\\\partial^2 f}{\\\\partial x \\\\partial y} \\\\\\\\\\n\\\\frac{\\\\partial^2 f}{\\\\partial y \\\\partial x} & \\\\frac{\\\\partial^2 f}{\\\\partial y^2}\\n\\\\end{bmatrix}\\n= \\\\begin{bmatrix}\\n2 & 0 \\\\\\\\\\n0 & 2\\n\\\\end{bmatrix}$\\n\\nMatriks Hessian adalah malar, jadi ia adalah sama pada setiap titik, termasuk titik kritikal $(0, 0)$. Untuk mengklasifikasikan titik kritikal, kita perlu menganalisis nilai eigen bagi matriks Hessian. Nilai eigen bagi $H(f)$ ialah penyelesaian kepada persamaan ciri:\\n\\n$|H(f) - \\\\lambda I| = \\\\begin{vmatrix}\\n2 - \\\\lambda & 0 \\\\\\\\\\n0 & 2 - \\\\lambda\\n\\\\end{vmatrix}\\n= (2 - \\\\lambda)^2$\\n\\nPersamaan ini mempunyai punca berganda pada $\\\\lambda = 2$. Oleh kerana kedua-dua nilai eigen adalah positif, matriks Hessian adalah positif pasti, yang bermaksud bahawa titik kritikal $(0, 0)$ ialah minimum tempatan.\\n\\nKesimpulannya, fungsi $f(x, y) = x^2 + y^2$ mempunyai satu titik kritikal pada $(0, 0)$, iaitu minimum tempatan.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Sudah tentu, berikut ialah masalah berangka untuk diselesaikan oleh pelajar matematik:\\n\\nCari kecerunan garis tangen kepada lengkung y = 2x^3 - 4 pada titik di mana x = 2.[/INST]Untuk mencari kecerunan garis tangen kepada lengkung y = 2x^3 - 4 pada titik di mana x = 2, kita perlu mencari terbitan fungsi berkenaan dengan x dan kemudian menilai pada x = 2.\\n\\nFungsinya ialah y = 2x^3 - 4.\\n\\nPertama, cari derivatif dy/dx:\\ndy/dx = d(2x^3 - 4)/dx = 6x^2 (menggunakan peraturan kuasa)\\n\\nSekarang, nilaikan terbitan pada x = 2:\\ndy/dx (x=2) = 6(2)^2 = 6(4) = 24\\n\\nJadi, kecerunan garis tangen kepada lengkung y = 2x^3 - 4 pada titik di mana x = 2 ialah 24.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Katakan kita mempunyai dua set A dan B di mana A = {2, 4, 6, 8, 10} dan B = {3, 6, 9, 12}. Cari persilangan dan gabungan dua set itu.[/INST]Untuk mencari persilangan set A dan B, kita mencari unsur-unsur yang biasa bagi kedua-dua set. Dalam kes ini, persimpangan adalah:\\n\\nA ∩ B = {6}\\n\\nUntuk mencari gabungan set A dan B, kami menggabungkan semua elemen daripada kedua-dua set, tanpa mengulangi sebarang unsur. Dalam kes ini, kesatuan adalah:\\n\\nA ∪ B = {2, 3, 4, 6, 8, 9, 10, 12}</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Terdapat 10 bola merah yang sama dan 5 bola biru yang sama. Dalam berapa banyak cara ia boleh diagihkan di antara 3 kotak yang berbeza supaya tiada kotak yang kosong?[/INST]Untuk menyelesaikan masalah ini, kita boleh menggunakan konsep bintang dan bar. Kami akan mula mengedarkan bola merah dan kemudian bola biru.\\n\\nUntuk bola merah, kerana terdapat 10 bola yang sama dan 3 kotak yang berbeza, dan tiada kotak boleh kosong, kita boleh menggunakan kaedah bintang dan bar. Kami akan mempunyai bola 10 - 3 = 7 \"tambahan\" untuk diedarkan di antara 3 kotak. Kita boleh mewakili ini sebagai 7 bintang dan 2 bar (untuk memisahkan 3 kotak). Bilangan cara untuk mengatur bintang dan bar ini diberikan oleh pekali binomial:\\n\\nC (n + k - 1, k - 1) = c (7 + 2, 2) = c (9, 2) = 9! / (2! * 7!) = 36 cara\\n\\nSekarang, untuk bola biru, terdapat 5 bola yang sama dan 3 kotak yang berbeza. Oleh kerana tiada kotak boleh kosong, kami akan mempunyai bola 5 - 3 = 2 \"tambahan\" untuk diedarkan di antara 3 kotak. Kita boleh mewakili ini sebagai 2 bintang dan 2 bar. Bilangan cara untuk mengatur bintang dan bar ini diberikan oleh pekali binomial:\\n\\nC (n + k - 1, k - 1) = c (2 + 2, 2) = c (4, 2) = 4! / (2! * 2!) = 6 cara\\n\\nAkhirnya, untuk mencari jumlah cara untuk mengedarkan kedua -dua bola merah dan biru, kami melipatgandakan bilangan cara untuk setiap warna:\\n\\n36 cara (bola merah) * 6 cara (bola biru) = 216 cara\\n\\nOleh itu, terdapat 216 cara untuk mengedarkan 10 bola merah dan 5 bola biru di antara 3 kotak yang berbeza supaya tiada kotak kosong.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Berikut ialah contoh soalan untuk diselesaikan oleh pelajar matematik berkaitan dengan polinomial pemfaktoran:\\n\\nFaktorkan polinomial x^2 + 9x + 14.[/INST]Untuk memfaktorkan polinomial x^2 + 9x + 14, kita perlu mencari dua binomial yang hasil darabnya bersamaan dengan polinomial yang diberikan. Kita boleh melakukan ini dengan mencari dua nombor yang didarab kepada 14 (sebutan malar) dan menambah hingga 9 (pekali bagi sebutan linear).\\n\\nDua nombor yang memenuhi syarat ini ialah 2 dan 7, kerana 2 * 7 = 14 dan 2 + 7 = 9.\\n\\nJadi, bentuk polinomial berfaktor ialah (x + 2)(x + 7).</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Sebuah syarikat pembuatan mengeluarkan dua produk, A dan B. Keuntungan seunit produk A ialah $20, dan keuntungan seunit produk B ialah $30. Syarikat mesti mengeluarkan sekurang-kurangnya 500 unit produk A dan 300 unit produk B, tetapi ia tidak boleh menghasilkan lebih daripada 800 unit secara keseluruhan. Selain itu, syarikat itu mempunyai kapasiti pengeluaran sebanyak 1000 unit sebulan. Menggunakan carian tabu, berapakah jumlah keuntungan maksimum yang boleh diperoleh oleh syarikat dalam sebulan?[/INST]Untuk menyelesaikan masalah ini menggunakan carian tabu, kami akan mentakrifkan fungsi objektif, kekangan, dan ruang carian terlebih dahulu. Kemudian, kami akan melakukan algoritma carian tabu untuk mencari penyelesaian yang optimum.\\n\\nFungsi objektif: Maksimumkan keuntungan = 20 * A + 30 * B\\n\\nKekangan:\\n1. A >= 500\\n2. B >= 300\\n3. A + B <= 800\\n4. A + B <= 1000 (kapasiti pengeluaran)\\n\\nRuang carian: Semua kemungkinan gabungan A dan B yang memenuhi kekangan.\\n\\nSekarang, mari kita lakukan algoritma carian tabu:\\n\\n1. Mulakan penyelesaian rawak dalam ruang carian, cth., A = 500, B = 300.\\n2. Buat senarai tabu kosong.\\n3. Hasilkan semua penyelesaian jiran dengan menukar A dan B dengan jumlah yang kecil (cth., +/- 10 unit).\\n4. Nilaikan fungsi objektif bagi setiap penyelesaian jiran dan semak sama ada ia memenuhi kekangan.\\n5. Pilih penyelesaian jiran terbaik yang tiada dalam senarai tabu.\\n6. Kemas kini senarai tabu dengan menambah penyelesaian semasa dan mengalih keluar penyelesaian tertua jika senarai melebihi saiz yang telah ditetapkan (cth., 5).\\n7. Ulangi langkah 3-6 untuk bilangan lelaran yang telah ditetapkan atau sehingga penyelesaian menumpu.\\n\\nSelepas menjalankan algoritma carian tabu, kami mendapati penyelesaian optimum ialah A = 500 dan B = 300, iaitu maksimum yang dibenarkan oleh kekangan. Jumlah keuntungan maksimum yang boleh diperoleh oleh syarikat dalam sebulan ialah:\\n\\nKeuntungan = 20 * A + 30 * B = 20 * 500 + 30 * 300 = 10,000 + 9,000 = $19,000.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Pasti, inilah masalah berangka bagi pelajar untuk menyelesaikannya:\\n\\nCari cerun garis tangen ke lengkung y = 2x^3 + 5x^2 - 3 pada titik (2, 29).[/INST]Untuk mencari kecerunan garis tangen kepada lengkung y = 2x^3 + 5x^2 - 3 pada titik (2, 29), kita perlu mencari terbitan bagi fungsi berkenaan dengan x dan kemudian menilai pada x = 2.\\n\\nTerbitan y berkenaan dengan x ialah:\\n\\ndy/dx = d(2x^3 + 5x^2 - 3)/dx\\n\\nMenggunakan peraturan kuasa untuk pembezaan, kita dapat:\\n\\ndy/dx = 6x^2 + 10x\\n\\nSekarang, kita perlu menilai derivatif pada x = 2:\\n\\ndy/dx(2) = 6(2)^2 + 10(2) = 6(4) + 20 = 24 + 20 = 44\\n\\nJadi, kecerunan garis tangen kepada lengkung y = 2x^3 + 5x^2 - 3 pada titik (2, 29) ialah 44.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Sebuah syarikat pelancongan mendakwa bahawa kebarangkalian untuk melihat ikan paus ialah 0.4 dan kebarangkalian untuk melihat gunung ais ialah 0.6 pada salah satu perjalanan mereka. Apakah kebarangkalian untuk melihat ikan paus dan gunung ais dalam perjalanan yang sama?[/INST]Untuk mencari kebarangkalian untuk melihat kedua-dua ikan paus dan gunung ais dalam perjalanan yang sama, kita perlu mendarabkan kebarangkalian individu bersama-sama, dengan mengandaikan bahawa peristiwa adalah bebas (bermaksud bahawa kejadian satu peristiwa tidak menjejaskan kejadian peristiwa yang lain) .\\n\\nKebarangkalian melihat ikan paus dan gunung ais = (kebarangkalian melihat ikan paus) × (kebarangkalian melihat gunung ais)\\n\\n= 0.4 × 0.6\\n= 0.24\\n\\nJadi, kebarangkalian untuk melihat ikan paus dan gunung ais dalam perjalanan yang sama ialah 0.24 atau 24%.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Sudah tentu, berikut ialah masalah berangka untuk diselesaikan oleh pelajar matematik:\\n\\nDiberi fungsi f(x) = x^3 - 2x^2 + 3x - 1, anggaran nilai terbitan f(x) pada x = 2 menggunakan kaedah beza hadapan dengan saiz langkah h = 0.5. Juga, gunakan peraturan trapezoid untuk menganggar nilai kamiran pasti f(x) daripada x = 1 hingga x = 2 menggunakan saiz langkah h = 0.25.[/INST]Mula-mula, mari kita anggarkan nilai terbitan f(x) pada x = 2 menggunakan kaedah beza hadapan dengan saiz langkah h = 0.5.\\n\\nFormula kaedah perbezaan hadapan ialah:\\n\\nf\\'(x) ≈ (f(x + h) - f(x)) / h\\n\\nMari kita hitung f(2) dan f(2.5):\\n\\nf(2) = (2^3) - 2(2^2) + 3(2) - 1 = 8 - 8 + 6 - 1 = 5\\nf(2.5) = (2.5^3) - 2(2.5^2) + 3(2.5) - 1 = 15.625 - 12.5 + 7.5 - 1 = 9.625\\n\\nSekarang, mari kita gunakan formula kaedah perbezaan hadapan:\\n\\nf\\'(2) ≈ (f(2.5) - f(2)) / 0.5 = (9.625 - 5) / 0.5 = 4.625 / 0.5 = 9.25\\n\\nSekarang, mari kita gunakan peraturan trapezoid untuk menganggar nilai kamiran pasti f(x) daripada x = 1 hingga x = 2 menggunakan saiz langkah h = 0.25.\\n\\nRumus peraturan trapezoid ialah:\\n\\n∫(a hingga b) f(x) dx ≈ (h/2) * (f(a) + 2 * Σf(x_i) + f(b))\\n\\ndi mana a = 1, b = 2, h = 0.25, dan x_i ialah titik antara a dan b dengan saiz langkah h.\\n\\nMari kita hitung f(1), f(1.25), f(1.5), f(1.75), dan f(2):\\n\\nf(1) = (1^3) - 2(1^2) + 3(1) - 1 = 1 - 2 + 3 - 1 = 1\\nf(1.25) = (1.25^3) - 2(1.25^2) + 3(1.25) - 1 = 1.953125 - 3.125 + 3.75 - 1 = 1.578125\\nf(1.5) = (1.5^3) - 2(1.5^2) + 3(1.5) - 1 = 3.375 - 4.5 + 4.5 - 1 = 2.375\\nf(1.75) = (1.75^3) - 2(1.75^2) + 3(1.75) - 1 = 5.359375 - 6.125 + 5.25 - 1 = 3.484375\\n\\nSekarang, mari kita gunakan formula peraturan trapezoid:\\n\\n∫(1 hingga 2) f(x) dx ≈ (0.25/2) * (f(1) + 2 * (f(1.25) + f(1.5) + f(1.75)) + f(2))\\n= (0.25/2) * (1 + 2 * (1.578125 + 2.375 + 3.484375) + 5)\\n= (0.125) * (1 + 2 * (7.4375) + 5)\\n= 0.125 * (1 + 14.875 + 5)\\n= 0.125 * 20.875\\n= 2.609375\\n\\nJadi, nilai anggaran terbitan f(x) pada x = 2 menggunakan kaedah beza hadapan dengan saiz langkah h = 0.5 ialah 9.25, dan nilai anggaran kamiran pasti f(x) daripada x = 1 kepada x = 2 menggunakan petua trapezoid dengan saiz langkah h = 0.25 ialah 2.609375.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Apakah nombor perdana terkecil yang boleh dinyatakan sebagai hasil tambah dua nombor perdana dengan tepat 4 cara?[/INST]Untuk mencari nombor perdana terkecil yang boleh dinyatakan sebagai hasil tambah dua nombor perdana dengan tepat 4 cara, kita boleh mulakan dengan menyenaraikan nombor perdana dan menyemak jumlahnya.\\n\\nNombor perdana: 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, ...\\n\\nMari kita mula menyemak jumlah:\\n\\n1. 7 (2+5) - hanya 1 hala\\n2. 11 (2+9) - hanya 1 hala\\n3. 13 (2+11) - hanya 1 hala\\n4. 17 (2+15, 7+10) - 2 cara\\n5. 19 (2+17) - hanya 1 hala\\n6. 23 (2+21, 5+18, 7+16) - 3 cara\\n7. 29 (2+27, 5+24, 11+18, 13+16) - 4 cara\\n\\nNombor perdana terkecil yang boleh dinyatakan sebagai hasil tambah dua nombor perdana dengan tepat 4 cara ialah 29. 4 cara tersebut ialah:\\n\\n1. 29 = 2 + 27\\n2. 29 = 5 + 24\\n3. 29 = 11 + 18\\n4. 29 = 13 + 16</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Di sebuah pekan kecil, terdapat 20 petani yang berkongsi tanah ragut yang sama. Setiap penternak mempunyai sekumpulan 8 ekor lembu. Jumlah maksimum lembu yang boleh ditampung di tanah ialah 100 ekor, tetapi jika lebih daripada 100 ekor lembu meragut, tanah akan menjadi terlalu ragut dan padang rumput akan rosak. Setiap petani boleh memilih berapa banyak lembu untuk meragut di atas tanah. Jika seorang petani tidak meragut mana-mana lembu di atas tanah, dia tidak akan menerima apa-apa keuntungan. Jika seorang petani meragut lembu di atas tanah, dia akan menerima keuntungan $400 setiap lembu pada akhir tahun. Walau bagaimanapun, jika petani secara kolektif meragut lebih daripada 100 ekor lembu, ragut yang berlebihan akan menyebabkan tanah menjadi musnah, dan setiap petani akan kehilangan $1000 setiap lembu.\\n\\nJika setiap penternak memilih bilangan lembu untuk meragut secara bebas dan merancang untuk memaksimumkan keuntungan sendiri, berapa jumlah ekor lembu yang akan meragut tanah secara keseluruhan dan berapakah jumlah keuntungan semua penternak pada akhir tahun?[/INST]Setiap penternak akan cuba memaksimumkan keuntungan mereka dengan meragut sebanyak mungkin lembu tanpa menyebabkan ragut berlebihan. Oleh kerana terdapat 20 orang petani dan tanahnya boleh menampung 100 ekor lembu, setiap petani boleh meragut 100/20 = 5 ekor lembu tanpa menyebabkan terlalu ragut.\\n\\nJika setiap petani meragut 5 ekor lembu, akan ada sejumlah 20 * 5 = 100 ekor lembu yang meragut tanah. Ini adalah bilangan maksimum lembu yang boleh dikekalkan oleh tanah tanpa rosak.\\n\\nJumlah keuntungan bagi setiap penternak ialah 5 ekor lembu * $400 setiap lembu = $2000. Oleh kerana terdapat 20 petani, jumlah keuntungan untuk semua petani pada akhir tahun ialah 20 * $2000 = $40,000.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Apakah kebarangkalian pecah rumah berlaku di kawasan kejiranan tertentu pada hari tertentu jika data sejarah menunjukkan bahawa terdapat purata 3 pecah rumah setiap minggu pada tahun lalu?[/INST]Untuk mencari kebarangkalian pecah rumah berlaku di kawasan kejiranan pada hari tertentu, kita perlu mencari purata bilangan pecah rumah setiap hari. Oleh kerana terdapat 3 kes pecah rumah setiap minggu dan 7 hari dalam seminggu, kami boleh mengira purata bilangan pecah rumah setiap hari seperti berikut:\\n\\nPurata pecah rumah sehari = Jumlah pecah rumah seminggu / Bilangan hari dalam seminggu\\nPurata pecah rumah sehari = 3/7\\n\\nSekarang, kita boleh menggunakan taburan Poisson untuk mencari kebarangkalian pecah rumah berlaku pada hari tertentu. Taburan Poisson digunakan untuk memodelkan bilangan peristiwa (dalam kes ini, pecah rumah) yang berlaku dalam selang masa atau ruang tetap. Formula untuk taburan Poisson ialah:\\n\\nP(X=k) = (e^(-λ) * λ^k) / k!\\n\\ndi mana:\\n- P(X=k) ialah kebarangkalian k kejadian berlaku\\n- λ (lambda) ialah purata bilangan peristiwa (dalam kes ini, pecah rumah setiap hari)\\n- k ialah bilangan peristiwa yang kita ingin cari kebarangkalian (dalam kes ini, 1 pecah rumah)\\n- e ialah asas logaritma semula jadi (kira-kira 2.71828)\\n\\nDalam kes ini, λ = 3/7, dan kami ingin mencari kebarangkalian 1 kecurian berlaku pada hari tertentu, jadi k = 1.\\n\\nP(X=1) = (e^(-3/7) * (3/7)^1) / 1!\\nP(X=1) = (2.71828^(-3/7) * (3/7)) / 1\\nP(X=1) ≈ 0.298\\n\\nJadi, kebarangkalian pecah rumah berlaku di kawasan kejiranan pada hari tertentu adalah lebih kurang 0.298 atau 29.8%.</s><s>[SYSTEM_PROMPT]You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is 2025-05-31.\\n\\nWhen you\\'re not sure about some information, you say that you don\\'t have the information and don\\'t make up anything.\\nIf the user\\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")[/SYSTEM_PROMPT][INST]Fungsi f(x) = x^3 - 6x^2 + 9x mempunyai dua titik genting. Cari titik infleksi bagi fungsi tersebut.[/INST]Untuk mencari titik infleksi fungsi, mula-mula kita perlu mencari derivatif kedua fungsi dan kemudian menyelesaikan titik di mana terbitan kedua adalah sama dengan sifar atau tidak ditentukan.\\n\\nFungsi yang diberikan: f(x) = x^3 - 6x^2 + '"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(dataset[0]['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae1e242f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
